'''
Created on 02.12.2013

@author: Aleks
'''

import re
import os
import trunk.data
from io import open
from trunk.src.Lemmatizer import Lemmatizer
from trunk.src.read_dictionary import read_dictionary, read_tags

def annotate_gold(dict, spanish):
    mypath = os.path.join(trunk.data.__path__[0], "usp", "Evaluation_corpus")
    os.chdir(mypath)
    #files = ["14.final", "16.final", "20.final", "22.final", "23.final"]
    files = ["14.final"]
    dict_word = 0
    spanish_word = 0
    all_words = 0
    lines = 0
    
    print "Going through files.."
    with open("gold_standard.txt", "w", encoding="utf8") as g:
        for file1 in files:
            with open(file1, encoding="utf8") as f:
                for line in f:
                    
                    
                    
                    if line.startswith("\\t"):
                        lines+=1
                        if lines>101:
                            break
                        g.write(line)
                        line = re.sub(ur'[\d:/,.?!+\xbf\xa1\(\)]+', "", line.strip()[3:len(line)-2]).lower().split()
                        g.write(u"\\S")
                        for word in line:
                            all_words += 1
                            if word in dict:
                                dict_word += 1
                                g.write(" %s/%s" % (word, dict[word][0]))
                            elif word in spanish:
                                spanish_word += 1
                                g.write(" %s/ESP" % word)
                            else:
                                g.write(u" -")
                        g.write(u"\n")
                    else:
                        g.write(line)
    print "Done.\n"
    
    print "Total words:\t\t", all_words
    print "spanish words:\t\t", spanish_word
    print "Dict words:\t\t", dict_word
    print "Total found words:\t", dict_word + spanish_word
                  
                
                
def main():
    lemm = Lemmatizer()
    
    print "Reading in data..."
    spanish = lemm.read_spanish()
    tags = read_tags(os.path.join(trunk.data.__path__[0], 'tags.txt'))
    dict = read_dictionary(os.path.join(trunk.data.__path__[0], 'dictionary.txt'), tags)
    print "Reading in done.\n"
    annotate_gold(dict, spanish)
    
if __name__ == '__main__':
    main()